{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "DNN_MNIST",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/sanglee/BIML2019/blob/master/DNN_MNIST.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "outputId": "962165ba-1270-453d-e6bc-6bc2de5d3af1",
        "id": "cWpUG4RuRoCu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        }
      },
      "cell_type": "code",
      "source": [
        "'''\n",
        "DNN to classify MNIST handwritten digits\n",
        "'''\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.examples.tutorials.mnist import input_data\n",
        "\n",
        "mnist = input_data.read_data_sets(\"./data/mnist\", one_hot=True)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Extracting ./data/mnist/train-images-idx3-ubyte.gz\n",
            "Extracting ./data/mnist/train-labels-idx1-ubyte.gz\n",
            "Extracting ./data/mnist/t10k-images-idx3-ubyte.gz\n",
            "Extracting ./data/mnist/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "76pNFt-XVdwD",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Parameters\n",
        "learning_rate = 0.001\n",
        "n_epochs = 15\n",
        "batch_size = 100\n",
        "display_step = 1\n",
        "\n",
        "# tf Graph input\n",
        "X = tf.placeholder(tf.float32, [batch_size, n_input])\n",
        "Y = tf.placeholder(tf.float32, [batch_size, n_classes])\n",
        "\n",
        "layer_1 = tf.layers.dense(X, units=256, activation='relu',\n",
        "                                         kernel_initializer=tf.contrib.layers.xavier_initializer(),\n",
        "                                         bias_initializer=tf.zeros_initializer())\n",
        "\n",
        "# Output layer with linear activation\n",
        "pred = tf.layers.dense(layer_1, units=10, activation=None,\n",
        "                                             kernel_initializer=tf.contrib.layers.xavier_initializer(),\n",
        "                                             bias_initializer=tf.zeros_initializer())\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "9zJZo1NsQsZJ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "# Define loss and optimizer\n",
        "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=Y))\n",
        "\n",
        "#optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)\n",
        "optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate).minimize(loss)\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "lsNywshJVz0p",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 319
        },
        "outputId": "d754786a-1e95-47d1-b563-48bf5b7c49bf"
      },
      "cell_type": "code",
      "source": [
        "# Launch the graph\n",
        "with tf.Session() as sess:\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "\n",
        "    # Training \n",
        "    for i in range(n_epochs):\n",
        "        total_loss = 0.\n",
        "        n_batches = int(mnist.train.num_examples/batch_size)\n",
        "        # Loop over all batches\n",
        "        for j in range(n_batches):\n",
        "            X_batch, Y_batch = mnist.train.next_batch(batch_size)\n",
        "            # Run optimization op (backprop) and cost op (to get loss value)\n",
        "            _, l = sess.run([optimizer, loss], feed_dict={X: X_batch, Y: Y_batch})\n",
        "            # Compute average loss\n",
        "            total_loss += l\n",
        "        # Display logs per epoch step\n",
        "        print('Average loss epoch {0}: {1}'.format(i, total_loss/n_batches))\n",
        "\n",
        "    print(\"Optimization Finished!\")\n",
        "\n",
        "\n",
        "    correct_preds = tf.equal(tf.argmax(pred, axis=1), tf.argmax(Y, axis=1))\n",
        "    accuracy = tf.reduce_sum(tf.cast(correct_preds, tf.float32))\n",
        "    \n",
        "    n_batches = int(mnist.test.num_examples/batch_size)\n",
        "    total_correct_preds = 0\n",
        "    \n",
        "    for i in range(n_batches):\n",
        "        X_batch, Y_batch = mnist.test.next_batch(batch_size)\n",
        "        accuracy_batch = sess.run(accuracy, feed_dict={X: X_batch, Y:Y_batch}) \n",
        "        total_correct_preds += accuracy_batch   \n",
        "    \n",
        "    print('Accuracy {0}'.format(total_correct_preds/mnist.test.num_examples))\n"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Average loss epoch 0: 0.5826158970594406\n",
            "Average loss epoch 1: 0.13547340057112953\n",
            "Average loss epoch 2: 0.0893200430426408\n",
            "Average loss epoch 3: 0.06601891248402271\n",
            "Average loss epoch 4: 0.05097045608880845\n",
            "Average loss epoch 5: 0.040394074181094766\n",
            "Average loss epoch 6: 0.032838352131657304\n",
            "Average loss epoch 7: 0.02575855269805867\n",
            "Average loss epoch 8: 0.02163409067751755\n",
            "Average loss epoch 9: 0.017588728840399363\n",
            "Average loss epoch 10: 0.01435625290598678\n",
            "Average loss epoch 11: 0.011398535867020573\n",
            "Average loss epoch 12: 0.009275353963920763\n",
            "Average loss epoch 13: 0.007153769477656831\n",
            "Average loss epoch 14: 0.005854526033733484\n",
            "Optimization Finished!\n",
            "Accuracy 0.9823\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "9yF2ip3gV4Ml",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}